/**
 * @author Fang Lu, fanglupku@gmail.com
 * 2011-5-31
 */
package code.extract;

import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class SpamFilter {
	/**
	 * text ratio in a tag
	 */
	private static double SPAM_RATIO_THRESHOLD = 0.5;
	/**
	 * a tag count
	 */
	private static int SPAM_ANCHOR_THRESHOLD = 5;
	private static Pattern linkPattern = Pattern.compile("((<a\\s+.*?>)|(<a\\s*>)).*?((</a\\s*>)|(</a\\s+.*?>))");
	public static boolean isSpamItem(String rawText){
		if (rawText == null){
			return true;
		}
		boolean result = false;
		String pureText = new String(BasePageDealer.eraseAllTags(rawText));
		pureText = pureText.replaceAll("\\s", "").replaceAll("&[#a-z0-9]{3,7};", "_");
		StringBuffer sb = new StringBuffer();
		Matcher matcher = linkPattern.matcher(rawText);
		
		int anchorCount = 0;
		while(matcher.find()){
			sb.append(BasePageDealer.eraseAllTags(rawText.substring(matcher.start(), matcher.end()))
					.replaceAll("\\s", "").replaceAll("&[#a-z0-9]{3,7};", "_"));
			++ anchorCount;
			
		}
//		System.out.println("++++++" + pureText);
//		System.out.println("======" + sb);
		if (pureText.length() == 0){
			result = true;
		}else if (anchorCount > SPAM_ANCHOR_THRESHOLD){
			result = true;
		}else if ((double)sb.length() / (double) pureText.length() > SPAM_RATIO_THRESHOLD){
			result = true;
		}
		return result;
	}
}
